import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from sklearn.model_selection import TimeSeriesSplit # https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.TimeSeriesSplit.html
from prophet import Prophet
from prophet.plot import plot_plotly, plot_components_plotly, plot_cross_validation_metric
from prophet.diagnostics import cross_validation, performance_metrics
import itertools
pd.options.display.float_format = '{:,.2f}'.format
# loading both prices and GoogleData csv files
prices_orig = pd.read_csv("prices.csv")
google_orig = pd.read_csv("GoogleData_oranges.csv")
# overview of the prices dataset structure, its head
prices_orig.head()
| Date | AveragePrice | TotalVolume | 4046 | 4225 | 4770 | TotalBags | SmallBags | LargeBags | XLargeBags | type | year | region | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2015-01-04 | 1.22 | 40,873.28 | 2,819.50 | 28,287.42 | 49.90 | 9,716.46 | 9,186.93 | 529.53 | 0.00 | conventional | 2015 | Albany |
| 1 | 2015-01-04 | 1.00 | 435,021.49 | 364,302.39 | 23,821.16 | 82.15 | 46,815.79 | 16,707.15 | 30,108.64 | 0.00 | conventional | 2015 | Atlanta |
| 2 | 2015-01-04 | NaN | 788,025.06 | 53,987.31 | 552,906.04 | 39,995.03 | 141,136.68 | 137,146.07 | 3,990.61 | 0.00 | conventional | 2015 | BaltimoreWashington |
| 3 | 2015-01-04 | 1.01 | 80,034.32 | 44,562.12 | 24,964.23 | 2,752.35 | 7,755.62 | 6,064.30 | 1,691.32 | 0.00 | conventional | 2015 | Boise |
| 4 | 2015-01-04 | 1.02 | 491,738.00 | 7,193.87 | 396,752.18 | 128.82 | 87,663.13 | 87,406.84 | 256.29 | 0.00 | conventional | 2015 | Boston |
# basic info of the prices dataset
prices_orig.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 25161 entries, 0 to 25160 Data columns (total 13 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Date 25161 non-null object 1 AveragePrice 24259 non-null float64 2 TotalVolume 24212 non-null float64 3 4046 24261 non-null float64 4 4225 24255 non-null float64 5 4770 24245 non-null float64 6 TotalBags 24271 non-null float64 7 SmallBags 24227 non-null float64 8 LargeBags 24214 non-null float64 9 XLargeBags 24293 non-null float64 10 type 25161 non-null object 11 year 25161 non-null int64 12 region 25161 non-null object dtypes: float64(9), int64(1), object(3) memory usage: 2.5+ MB
# changing the datatype of the date column
prices_orig = prices_orig.astype({'Date': 'datetime64[ns]'})
# prices dataset overview
prices_orig.describe(include="all")
/Users/radimmusalek/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
| Date | AveragePrice | TotalVolume | 4046 | 4225 | 4770 | TotalBags | SmallBags | LargeBags | XLargeBags | type | year | region | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 25161 | 24,259.00 | 24,212.00 | 24,261.00 | 24,255.00 | 24,245.00 | 24,271.00 | 24,227.00 | 24,214.00 | 24,293.00 | 25161 | 25,161.00 | 25161 |
| unique | 233 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 2 | NaN | 54 |
| top | 2016-10-23 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | conventional | NaN | Boise |
| freq | 108 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | 12582 | NaN | 466 |
| first | 2015-01-04 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| last | 2019-07-14 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| mean | NaN | 1.40 | 917,091.34 | 298,153.02 | 295,828.43 | 22,463.62 | 295,043.21 | 212,419.88 | 75,899.24 | 4,405.64 | NaN | 2,016.78 | NaN |
| std | NaN | 0.38 | 3,731,937.46 | 1,288,583.40 | 1,209,444.82 | 104,662.07 | 1,232,967.42 | 869,304.26 | 358,607.93 | 25,806.50 | NaN | 1.32 | NaN |
| min | NaN | 0.44 | 84.56 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | NaN | 2,015.00 | NaN |
| 25% | NaN | 1.11 | 13,234.04 | 822.59 | 3,061.14 | 0.00 | 7,319.51 | 4,770.44 | 235.82 | 0.00 | NaN | 2,016.00 | NaN |
| 50% | NaN | 1.36 | 119,052.77 | 9,855.11 | 26,977.55 | 193.76 | 46,257.71 | 31,337.34 | 4,043.68 | 0.00 | NaN | 2,017.00 | NaN |
| 75% | NaN | 1.64 | 473,378.30 | 114,121.88 | 149,871.91 | 6,078.17 | 139,429.54 | 100,882.73 | 30,503.37 | 313.89 | NaN | 2,018.00 | NaN |
| max | NaN | 3.25 | 63,716,144.15 | 22,743,616.17 | 20,470,572.61 | 2,546,439.11 | 23,472,988.69 | 15,436,246.72 | 7,864,297.23 | 844,929.83 | NaN | 2,019.00 | NaN |
# interactive plot with conventional type prices per region over the time period
fig_pr_con = px.line(prices_orig[prices_orig["type"]=="conventional"],
x="Date", y="AveragePrice",
color="region", title='Average price of conventional per region')
fig_pr_con.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_pr_con.show()
# interactive plot with organic type prices per region over the time period
fig_pr_org = px.line(prices_orig[prices_orig["type"]=="organic"],
x="Date", y="AveragePrice",
color="region", title='Average price of organic per region')
fig_pr_org.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_pr_org.show()
# interactive plot with conventional type volumes per region over the time period
fig_vol_con = px.line(prices_orig[prices_orig["type"]=="conventional"],
x="Date", y="TotalVolume",
color="region", title="Total volumes of conventional per region")
fig_vol_con.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_vol_con.show()
# interactive plot with organic type volumes per region over the time period
fig_vol_org = px.line(prices_orig[prices_orig["type"]=="organic"],
x="Date", y="TotalVolume",
color="region", title="Total volumes of organic per region")
fig_vol_org.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig_vol_org.show()
# overview of the Google dataset structure, its head
google_orig.head()
| Unnamed: 0 | oranges: (United States) | organic: (United States) | oranges recipe: (United States) | oranges smoothie: (United States) | oranges salad: (United States) | organic oranges: (United States) | |
|---|---|---|---|---|---|---|---|
| 0 | 2015-01-04 | 46 | 76 | 84 | 8 | 51 | 10 |
| 1 | 2015-01-11 | 48 | 79 | 76 | 8 | 38 | 29 |
| 2 | 2015-01-18 | 48 | 82 | 84 | 9 | 66 | 27 |
| 3 | 2015-01-25 | 49 | 82 | 89 | 8 | 44 | 28 |
| 4 | 2015-02-01 | 52 | 81 | 82 | 7 | 46 | 42 |
# basic info about the Google dataset
google_orig.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 239 entries, 0 to 238 Data columns (total 7 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Unnamed: 0 239 non-null object 1 oranges: (United States) 239 non-null int64 2 organic: (United States) 239 non-null int64 3 oranges recipe: (United States) 239 non-null int64 4 oranges smoothie: (United States) 239 non-null int64 5 oranges salad: (United States) 239 non-null int64 6 organic oranges: (United States) 239 non-null int64 dtypes: int64(6), object(1) memory usage: 13.2+ KB
# changing the datatype of the date column
google_orig = google_orig.astype({"Unnamed: 0": "datetime64[ns]"})
# Google dataset overview
google_orig.describe(include="all")
/Users/radimmusalek/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
| Unnamed: 0 | oranges: (United States) | organic: (United States) | oranges recipe: (United States) | oranges smoothie: (United States) | oranges salad: (United States) | organic oranges: (United States) | |
|---|---|---|---|---|---|---|---|
| count | 239 | 239.00 | 239.00 | 239.00 | 239.00 | 239.00 | 239.00 |
| unique | 239 | NaN | NaN | NaN | NaN | NaN | NaN |
| top | 2018-08-19 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| freq | 1 | NaN | NaN | NaN | NaN | NaN | NaN |
| first | 2015-01-04 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| last | 2019-07-28 00:00:00 | NaN | NaN | NaN | NaN | NaN | NaN |
| mean | NaN | 52.31 | 82.50 | 66.99 | 32.71 | 50.18 | 36.06 |
| std | NaN | 11.29 | 6.85 | 14.84 | 19.54 | 12.26 | 12.70 |
| min | NaN | 29.00 | 56.00 | 33.00 | 7.00 | 15.00 | 10.00 |
| 25% | NaN | 44.50 | 79.00 | 55.00 | 15.00 | 42.00 | 27.00 |
| 50% | NaN | 52.00 | 83.00 | 68.00 | 30.00 | 50.00 | 35.00 |
| 75% | NaN | 60.00 | 87.00 | 78.50 | 50.50 | 59.00 | 44.50 |
| max | NaN | 100.00 | 100.00 | 100.00 | 100.00 | 100.00 | 100.00 |
# renaming the columns to get more readable format
google_orig.columns = ["Date", "oranges", "organic",
"oranges recipe", "oranges smoothie",
"oranges salad", "organic oranges"]
# interactive plot with G-searches per keyword over the time period
fig3 = px.line(google_orig, x="Date", y=google_orig.columns,
title="Google searches overview", labels=dict(value="Relative number of searches"))
fig3.update_xaxes(
dtick="M6",
tickformat="%b-%y")
fig3.show()
# split the prices dataset into organic and conventional type
prices_US_conv = prices_orig[prices_orig["type"]=="conventional"].reset_index(drop=True)
prices_US_org = prices_orig[prices_orig["type"]=="organic"].reset_index(drop=True)
# grouping the regional data of the conventional type oranges into whole US by Date
# AveragePrice is mean of its values, the other columns are summarised
prices_US_conv_group = prices_US_conv.groupby("Date").agg({"AveragePrice":"mean",
"TotalVolume":"sum",
"4046":"sum",
"4225":"sum",
"4770":"sum",
"TotalBags":"sum",
"SmallBags":"sum",
"LargeBags":"sum",
"XLargeBags":"sum"
}).reset_index()
# same process as above done for the organic oranges data
prices_US_org_group = prices_US_org.groupby("Date").agg({"AveragePrice":"mean",
"TotalVolume":"sum",
"4046":"sum",
"4225":"sum",
"4770":"sum",
"TotalBags":"sum",
"SmallBags":"sum",
"LargeBags":"sum",
"XLargeBags":"sum"
}).reset_index()
# merging of the both grouped data with the google searches data
# we keep all the columns of the google dataset for both merges
prices_US_conv_merged = prices_US_conv_group.merge(google_orig, how="left", on="Date")
prices_US_org_merged = prices_US_org_group.merge(google_orig, how="left", on="Date")
# left join is used as the most significant parameters for the prediction will be the price and volumes
# since those are the ones we're interested in for our model
# one missing date for G-searches, no need to extrapolate since Prophet doesn't care,
# otherwise I could e.g. mean of the values around that day
# spitting the datasets on training part (till the end of 2018) and testing part (from beginning of 2019)
prices_US_conv_2015_2018 = prices_US_conv_merged[prices_US_conv_merged["Date"]<"2019-01-01"]
prices_US_conv_2019 = prices_US_conv_merged[prices_US_conv_merged["Date"]>="2019-01-01"]
prices_US_org_2015_2018 = prices_US_org_merged[prices_US_org_merged["Date"]<"2019-01-01"]
prices_US_org_2019 = prices_US_org_merged[prices_US_org_merged["Date"]>="2019-01-01"]
# double-check that the split is done correctly
print(prices_US_conv_2015_2018.iloc[-1][0])
print(prices_US_conv_2019.iloc[0][0])
print(prices_US_org_2015_2018.iloc[-1][0])
print(prices_US_org_2019.iloc[0][0])
2018-12-02 00:00:00 2019-01-07 00:00:00 2018-12-02 00:00:00 2019-01-07 00:00:00
# simplifing the dataframe for use by Prophet
conv_P_df = prices_US_conv_2015_2018[["Date", "AveragePrice"]]
conv_P_df.columns = ["ds", "y"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_conv_P_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_conv_P_def.fit(conv_P_df)
<prophet.forecaster.Prophet at 0x7fd528317250>
future_conv_P_def = model_conv_P_def.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_P_def = model_conv_P_def.predict(future_conv_P_def)
fig_conv_P_def = model_conv_P_def.plot(forecast_conv_P_def)
# model components visualisation
fig_conv_P_comp_def = model_conv_P_def.plot_components(forecast_conv_P_def)
# selecting only the test period forecast
pred_conv_P_2019_def = forecast_conv_P_def[forecast_conv_P_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_P_2019_def = prices_US_conv_2019[["Date", "AveragePrice"]]
actuals_conv_P_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_P_2019_def = actuals_conv_P_2019_def.merge(pred_conv_P_2019_def, on="ds")
# plotting training, test and prediction data
conv_P_df.set_index("ds")["y"].plot(label="training")
conv_P_2019_def.set_index("ds")["y"].plot(label="test")
conv_P_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - default model")
plt.legend()
plt.savefig("Price conventional - default model.png")
plt.show()
# Defining MAPE function
def MAPE(Y_actual,Y_Predicted):
mape = np.mean(np.abs((Y_actual - Y_Predicted)/Y_actual))*100
return mape
# model's MAPE calculation
model_conv_P_def_MAPE = MAPE(conv_P_2019_def["y"], conv_P_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_conv_P_def_MAPE))
Default model MAPE: 18.27
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"seasonality_mode": ["additive", "multiplicative"],
"growth": ["linear", "flat"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(conv_P_df) # Fit model with given params
df_cv_conv_P = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_conv_P = performance_metrics(df_cv_conv_P, rolling_window=1)
mapes.append(df_p_conv_P["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f1d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c00e090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8250> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52beb3710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0236d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f350> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad78c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f4d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0bc750> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528827250> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fc50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fdd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8250> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c00e190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f310> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b8306d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0bcd90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a892910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0bc790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a875710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a879810> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad78f50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a893850> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a892690> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a896f10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876cd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bedaf50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876a50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad8ca90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a876090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a8818d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52acfee50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad8ca90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad8c650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f510> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bedaf50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c086e10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c086e10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f4d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c00e090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad8cf50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c00e090> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b91acd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b8306d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad59490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad78590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b8306d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a893e50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8950> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b91acd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f4d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad8c650>
changepoint_prior_scale seasonality_prior_scale seasonality_mode growth \
0 0.00 0.01 additive linear
1 0.00 0.01 additive flat
2 0.00 0.01 multiplicative linear
3 0.00 0.01 multiplicative flat
4 0.00 0.10 additive linear
.. ... ... ... ...
59 0.50 1.00 multiplicative flat
60 0.50 10.00 additive linear
61 0.50 10.00 additive flat
62 0.50 10.00 multiplicative linear
63 0.50 10.00 multiplicative flat
daily_seasonality weekly_seasonality mape
0 False False 0.16
1 False False 0.11
2 False False 0.16
3 False False 0.11
4 False False 0.15
.. ... ... ...
59 False False 0.11
60 False False 0.23
61 False False 0.11
62 False False 0.30
63 False False 0.11
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_conv_P = all_params[np.argmin(mapes)]
print(best_params_conv_P)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 0.01, 'seasonality_mode': 'additive', 'growth': 'flat', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_conv_P = Prophet(**best_params_conv_P)
# fitting the model on the whole training set
model_conv_P.fit(conv_P_df)
<prophet.forecaster.Prophet at 0x7fd52c0bc090>
len(prices_US_conv_2019)
28
future_conv_P = model_conv_P.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_P = model_conv_P.predict(future_conv_P)
fig_conv_P = model_conv_P.plot(forecast_conv_P)
# model components visualisation
fig_conv_P_comp = model_conv_P.plot_components(forecast_conv_P)
# cross-validation of the selected model
df_cv_conv_P = cross_validation(model_conv_P, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cc2ea90>
# MAPE of the cross-validated model visualisation
fig_conv_P_cv_mape = plot_cross_validation_metric(df_cv_conv_P, metric='mape')
# selecting only the test period forecast
pred_conv_P_2019 = forecast_conv_P[forecast_conv_P["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_P_2019 = prices_US_conv_2019[["Date", "AveragePrice"]]
actuals_conv_P_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_P_2019 = actuals_conv_P_2019.merge(pred_conv_P_2019, on="ds")
# plotting training, test and prediction data
conv_P_df.set_index("ds")["y"].plot(label="training")
conv_P_2019.set_index("ds")["y"].plot(label="test")
conv_P_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - test vs. prediction")
plt.legend()
plt.savefig("Price conventional - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_conv_P_MAPE = MAPE(conv_P_2019["y"], conv_P_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_conv_P_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_conv_P_MAPE))
Default model MAPE: 18.27 Tuned model MAPE: 11.14
# simplifing the full dataset for use by Prophet
conv_P_full = prices_US_conv_merged[["Date", "AveragePrice"]]
conv_P_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month forecast
model_conv_P_for = Prophet(**best_params_conv_P)
model_conv_P_for.fit(conv_P_full)
future_6m_conv_P = model_conv_P_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_conv_P = model_conv_P_for.predict(future_6m_conv_P)
fig_6m_conv_P = model_conv_P_for.plot(forecast_6m_conv_P)
# plotting forecast
conv_P_full.set_index("ds")["y"].plot(label="original")
forecast_6m_conv_P.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price conventional - forecast")
plt.legend()
plt.savefig("Price conventional - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
org_P_df = prices_US_org_2015_2018[["Date", "AveragePrice"]]
org_P_df.columns = ["ds", "y"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_org_P_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_org_P_def.fit(org_P_df)
<prophet.forecaster.Prophet at 0x7fd52d6d7450>
future_org_P_def = model_org_P_def.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_P_def = model_org_P_def.predict(future_org_P_def)
fig_org_P_def = model_org_P_def.plot(forecast_org_P_def)
# model components visualisation
fig_org_P_comp_def = model_org_P_def.plot_components(forecast_org_P_def)
# selecting only the test period forecast
pred_org_P_2019_def = forecast_org_P_def[forecast_org_P_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_P_2019_def = prices_US_org_2019[["Date", "AveragePrice"]]
actuals_org_P_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_P_2019_def = actuals_org_P_2019_def.merge(pred_org_P_2019_def, on="ds")
# plotting training, test and prediction data
org_P_df.set_index("ds")["y"].plot(label="training")
org_P_2019_def.set_index("ds")["y"].plot(label="test")
org_P_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - default model")
plt.legend()
plt.savefig("Price organic - default model.png")
plt.show()
# model's MAPE calculation
model_org_P_def_MAPE = MAPE(org_P_2019_def["y"], org_P_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_org_P_def_MAPE))
Default model MAPE: 19.01
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"seasonality_mode": ["additive", "multiplicative"],
"growth": ["linear", "flat"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(org_P_df) # Fit model with given params
df_cv_org_P = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_org_P = performance_metrics(df_cv_org_P, rolling_window=1)
mapes.append(df_p_org_P["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b8435d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b827dd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd527e650d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528741ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7513d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bee0890> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52acd75d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd527cc9c50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b874cd0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cd48b50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d6d72d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7ad5d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528713510> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b827dd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a45c690> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fc50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fc50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5285a76d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d836b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c018710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cda7ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d81b050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bec8f10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528741ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ab66450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cbe5610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c229310> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d6d7110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c203fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0ec310> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad411d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52b83a3d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a8924d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a520c10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52acd75d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0f7610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a894190> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5236f40d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d777690> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a8962d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d75fc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5285a7550> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52be9f8d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5236e5c90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d75fc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd526d106d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d75fc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d836b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7f5250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52be7f790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ab9b510> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5236f40d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d836b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7ac050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a8efb10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7513d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d6301d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad0c810> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ba91390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7ac050> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd526d106d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace0c90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5236e50d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52adf6050>
changepoint_prior_scale seasonality_prior_scale seasonality_mode growth \
0 0.00 0.01 additive linear
1 0.00 0.01 additive flat
2 0.00 0.01 multiplicative linear
3 0.00 0.01 multiplicative flat
4 0.00 0.10 additive linear
.. ... ... ... ...
59 0.50 1.00 multiplicative flat
60 0.50 10.00 additive linear
61 0.50 10.00 additive flat
62 0.50 10.00 multiplicative linear
63 0.50 10.00 multiplicative flat
daily_seasonality weekly_seasonality mape
0 False False 0.10
1 False False 0.07
2 False False 0.11
3 False False 0.07
4 False False 0.11
.. ... ... ...
59 False False 0.07
60 False False 0.25
61 False False 0.07
62 False False 0.24
63 False False 0.07
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_org_P = all_params[np.argmin(mapes)]
print(best_params_org_P)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 0.01, 'seasonality_mode': 'additive', 'growth': 'flat', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_org_P = Prophet(**best_params_org_P)
# fitting the model on the whole training set
model_org_P.fit(org_P_df)
<prophet.forecaster.Prophet at 0x7fd52ccdba10>
len(prices_US_org_2019)
28
future_org_P = model_org_P.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_P = model_org_P.predict(future_org_P)
fig_org_P = model_org_P.plot(forecast_org_P)
# model components visualisation
fig_org_P_comp = model_org_P.plot_components(forecast_org_P)
# cross-validation of the selected model
df_cv_org_P = cross_validation(model_org_P, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52addddd0>
# MAPE of the cross-validated model visualisation
fig_org_P_cv_mape = plot_cross_validation_metric(df_cv_org_P, metric='mape')
# selecting only the test period forecast
pred_org_P_2019 = forecast_org_P[forecast_org_P["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_P_2019 = prices_US_org_2019[["Date", "AveragePrice"]]
actuals_org_P_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_P_2019 = actuals_org_P_2019.merge(pred_org_P_2019, on="ds")
# plotting training, test and prediction data
org_P_df.set_index("ds")["y"].plot(label="training")
org_P_2019.set_index("ds")["y"].plot(label="test")
org_P_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - test vs. prediction")
plt.legend()
plt.savefig("Price organic - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_org_P_MAPE = MAPE(org_P_2019["y"], org_P_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_org_P_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_org_P_MAPE))
Default model MAPE: 19.01 Tuned model MAPE: 8.35
# simplifing the full dataset for use by Prophet
org_P_full = prices_US_org_merged[["Date", "AveragePrice"]]
org_P_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_org_P_for = Prophet(**best_params_org_P)
model_org_P_for.fit(org_P_full)
future_6m_org_P = model_org_P_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_org_P = model_org_P_for.predict(future_6m_org_P)
fig_6m_org_P = model_org_P_for.plot(forecast_6m_org_P)
# plotting forecast
org_P_full.set_index("ds")["y"].plot(label="original")
forecast_6m_org_P.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Price")
plt.title("Price organic - forecast")
plt.legend()
plt.savefig("Price organic - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
conv_Q_df = prices_US_conv_2015_2018[["Date", "TotalVolume"]]
conv_Q_df.columns = ["ds", "y"]
# conv_Q_df["cap"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_conv_Q_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_conv_Q_def.fit(conv_Q_df)
<prophet.forecaster.Prophet at 0x7fd52cc20110>
future_conv_Q_def = model_conv_Q_def.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_Q_def = model_conv_Q_def.predict(future_conv_Q_def)
fig_conv_Q_def = model_conv_Q_def.plot(forecast_conv_Q_def)
# model components visualisation
fig_conv_Q_comp_def = model_conv_Q_def.plot_components(forecast_conv_Q_def)
# selecting only the test period forecast
pred_conv_Q_2019_def = forecast_conv_Q_def[forecast_conv_Q_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_Q_2019_def = prices_US_conv_2019[["Date", "TotalVolume"]]
actuals_conv_Q_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_Q_2019_def = actuals_conv_Q_2019_def.merge(pred_conv_Q_2019_def, on="ds")
# plotting training, test and prediction data
conv_Q_df.set_index("ds")["y"].plot(label="training")
conv_Q_2019_def.set_index("ds")["y"].plot(label="test")
conv_Q_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - default model")
plt.legend()
plt.savefig("Volume conventional - default model.png")
plt.show()
# model's MAPE calculation
model_conv_Q_def_MAPE = MAPE(conv_Q_2019_def["y"], conv_Q_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_conv_Q_def_MAPE))
Default model MAPE: 13.46
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"growth": ["linear", "flat"], # "logistic" might be better for this model but we'd need the cap value estimate from the client
"seasonality_mode": ["additive", "multiplicative"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(conv_Q_df) # Fit model with given params
df_cv_conv_Q = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_conv_Q = performance_metrics(df_cv_conv_Q, rolling_window=1)
mapes.append(df_p_conv_Q["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c00eb90> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbba490> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cd89850> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb6e290> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cc83350> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8928d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f902d50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8ca810> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cc83350> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a89f950> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a881b50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f902b50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52be7f790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f902d50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cc88910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb94ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb94ad0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fba1550> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f91da90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c0ec310> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52a881b50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c079510> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f806e50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50efdd450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f90e0d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c045750> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50efdd450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f923590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5283dcf50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f90e0d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50efdd450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52be34f90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f806e50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbba3d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d75fc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d6301d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8ca810> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb6e290> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528741610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bee0fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f902b50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52bee06d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfe10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d7a0110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd528741610> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c045750> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbba490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f806c90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fd90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d750150> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f902b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02fd90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd527e65a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f90e0d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f806c90>
changepoint_prior_scale seasonality_prior_scale growth seasonality_mode \
0 0.00 0.01 linear additive
1 0.00 0.01 linear multiplicative
2 0.00 0.01 flat additive
3 0.00 0.01 flat multiplicative
4 0.00 0.10 linear additive
.. ... ... ... ...
59 0.50 1.00 flat multiplicative
60 0.50 10.00 linear additive
61 0.50 10.00 linear multiplicative
62 0.50 10.00 flat additive
63 0.50 10.00 flat multiplicative
daily_seasonality weekly_seasonality mape
0 False False 0.16
1 False False 0.17
2 False False 0.18
3 False False 0.19
4 False False 0.15
.. ... ... ...
59 False False 0.17
60 False False 0.24
61 False False 0.24
62 False False 0.17
63 False False 0.17
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_conv_Q = all_params[np.argmin(mapes)]
print(best_params_conv_Q)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 10.0, 'growth': 'linear', 'seasonality_mode': 'multiplicative', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_conv_Q = Prophet(**best_params_conv_Q)
# fitting the model on the whole training set
model_conv_Q.fit(conv_Q_df)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
<prophet.forecaster.Prophet at 0x7fd50f9022d0>
future_conv_Q = model_conv_Q.make_future_dataframe(periods=len(prices_US_conv_2019), freq="W")
# forecast visualisation
forecast_conv_Q = model_conv_Q.predict(future_conv_Q)
fig_conv_Q = model_conv_Q.plot(forecast_conv_Q)
# model components visualisation
fig_conv_Q_comp = model_conv_Q.plot_components(forecast_conv_Q)
# cross-validation of the selected model
df_cv_conv_Q = cross_validation(model_conv_Q, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f806c90> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# MAPE of the cross-validated model visualisation
fig_conv_Q_cv_mape = plot_cross_validation_metric(df_cv_conv_Q, metric='mape')
# selecting only the test period forecast
pred_conv_Q_2019 = forecast_conv_Q[forecast_conv_Q["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_conv_Q_2019 = prices_US_conv_2019[["Date", "TotalVolume"]]
actuals_conv_Q_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
conv_Q_2019 = actuals_conv_Q_2019.merge(pred_conv_Q_2019, on="ds")
# plotting training, test and prediction data
conv_Q_df.set_index("ds")["y"].plot(label="training")
conv_Q_2019.set_index("ds")["y"].plot(label="test")
conv_Q_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - test vs. prediction")
plt.legend()
plt.savefig("Volume conventional - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_conv_Q_MAPE = MAPE(conv_Q_2019["y"], conv_Q_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_conv_Q_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_conv_Q_MAPE))
Default model MAPE: 13.46 Tuned model MAPE: 13.70
# simplifing the full dataset for use by Prophet
conv_Q_full = prices_US_conv_merged[["Date", "TotalVolume"]]
conv_Q_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_conv_Q_for = Prophet(**best_params_conv_Q)
model_conv_Q_for.fit(conv_Q_full)
future_6m_conv_Q = model_conv_Q_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_conv_Q = model_conv_Q_for.predict(future_6m_conv_Q)
fig_6m_conv_Q = model_conv_Q_for.plot(forecast_6m_conv_Q)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# plotting forecast
conv_Q_full.set_index("ds")["y"].plot(label="original")
forecast_6m_conv_Q.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume conventional - forecast")
plt.legend()
plt.savefig("Volume conventional - forecast.png")
plt.show()
# simplifing the dataframe for use by Prophet
org_Q_df = prices_US_org_2015_2018[["Date", "TotalVolume"]]
org_Q_df.columns = ["ds", "y"]
# org_Q_df["cap"]
# creating model with default values, skipping only the daily and weekly seasonality which isn't present in our data
model_org_Q_def = Prophet(daily_seasonality=False, weekly_seasonality=False)
# fitting the model on the whole training set
model_org_Q_def.fit(org_Q_df)
<prophet.forecaster.Prophet at 0x7fd510d81b50>
future_org_Q_def = model_org_Q_def.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_Q_def = model_org_Q_def.predict(future_org_Q_def)
fig_org_Q_def = model_org_Q_def.plot(forecast_org_Q_def)
# model components visualisation
fig_org_Q_comp_def = model_org_Q_def.plot_components(forecast_org_Q_def)
# selecting only the test period forecast
pred_org_Q_2019_def = forecast_org_Q_def[forecast_org_Q_def["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_Q_2019_def = prices_US_org_2019[["Date", "TotalVolume"]]
actuals_org_Q_2019_def.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_Q_2019_def = actuals_org_Q_2019_def.merge(pred_org_Q_2019_def, on="ds")
# plotting training, test and prediction data
org_Q_df.set_index("ds")["y"].plot(label="training")
org_Q_2019_def.set_index("ds")["y"].plot(label="test")
org_Q_2019_def.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - default model")
plt.legend()
plt.savefig("Volume organic - default model.png")
plt.show()
# model's MAPE calculation
model_org_Q_def_MAPE = MAPE(org_Q_2019_def["y"], org_Q_2019_def["yhat"])
print("Default model MAPE: {:.2f}".format(model_org_Q_def_MAPE))
Default model MAPE: 27.94
# cross-validation hyperparameters on the Prophet model
param_grid = {
"changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5],
"seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0],
"growth": ["linear", "flat"], # "logistic" might be better for this model but we'd need the cap value estimate from the client
"seasonality_mode": ["additive", "multiplicative"],
"daily_seasonality": [False],
"weekly_seasonality": [False]
}
# Generate all combinations of parameters
all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
mapes = [] # Store the MAPEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(org_Q_df) # Fit model with given params
df_cv_org_Q = cross_validation(m, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
df_p_org_Q = performance_metrics(df_cv_org_Q, rolling_window=1)
mapes.append(df_p_org_Q["mape"].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results["mape"] = mapes
print(tuning_results)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f90e0d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099ead0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fba7710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e160210> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099edd0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e1601d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb67110> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cbe5110> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d829f50> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad35bd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5109304d0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099ead0> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbf6710> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e8a6410> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d864790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb67fd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb67250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50ef91590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f888a10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfa90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cb99250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5109547d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e483b90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd523724d90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fc007d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c273390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f923e90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb77710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52cb99250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb67450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd510dbb910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb6e350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb67250> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099ead0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f690> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e0d0390> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad35790> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb6e350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbf6710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5108d5450> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfa90> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb49750> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50ef91710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fba1490> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb6e350> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd510d75f10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5108dedd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ad35bd0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e1601d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099ead0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52ace8b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50f8dfc10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd51099e650> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd5109106d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb77590> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f690> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c273b10> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52c02f6d0> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fb77710> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd52d829f50> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd523724910> INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50fbf6bd0>
changepoint_prior_scale seasonality_prior_scale growth seasonality_mode \
0 0.00 0.01 linear additive
1 0.00 0.01 linear multiplicative
2 0.00 0.01 flat additive
3 0.00 0.01 flat multiplicative
4 0.00 0.10 linear additive
.. ... ... ... ...
59 0.50 1.00 flat multiplicative
60 0.50 10.00 linear additive
61 0.50 10.00 linear multiplicative
62 0.50 10.00 flat additive
63 0.50 10.00 flat multiplicative
daily_seasonality weekly_seasonality mape
0 False False 0.12
1 False False 0.14
2 False False 0.36
3 False False 0.35
4 False False 0.11
.. ... ... ...
59 False False 0.38
60 False False 0.27
61 False False 0.22
62 False False 0.38
63 False False 0.38
[64 rows x 7 columns]
# selecting the best hyperparameters
best_params_org_Q = all_params[np.argmin(mapes)]
print(best_params_org_Q)
{'changepoint_prior_scale': 0.001, 'seasonality_prior_scale': 10.0, 'growth': 'linear', 'seasonality_mode': 'additive', 'daily_seasonality': False, 'weekly_seasonality': False}
# creating the model with the best parameters
model_org_Q = Prophet(**best_params_org_Q)
# fitting the model on the whole training set
model_org_Q.fit(org_Q_df)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
<prophet.forecaster.Prophet at 0x7fd50eef9b90>
future_org_Q = model_org_Q.make_future_dataframe(periods=len(prices_US_org_2019), freq="W")
# forecast visualisation
forecast_org_Q = model_org_Q.predict(future_org_Q)
fig_org_Q = model_org_Q.plot(forecast_org_Q)
# model components visualisation
fig_org_Q_comp = model_org_Q.plot_components(forecast_org_Q)
# cross-validation of the selected model
df_cv_org_Q = cross_validation(model_org_Q, initial="730 days", period="90 days", horizon="365 days", parallel="processes")
INFO:prophet:Making 4 forecasts with cutoffs between 2017-03-07 00:00:00 and 2017-12-02 00:00:00 INFO:prophet:Applying in parallel with <concurrent.futures.process.ProcessPoolExecutor object at 0x7fd50e6d7a10> WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton. WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# MAPE of the cross-validated model visualisation
fig_org_Q_cv_mape = plot_cross_validation_metric(df_cv_org_Q, metric='mape')
# selecting only the test period forecast
pred_org_Q_2019 = forecast_org_Q[forecast_org_Q["ds"]>"2019-01-01"][["ds", "yhat"]]
# simplifing the actuals dataframe for comparison
actuals_org_Q_2019 = prices_US_org_2019[["Date", "TotalVolume"]]
actuals_org_Q_2019.columns = ["ds", "y"]
# merging the prediction and actuals dataframes into one
org_Q_2019 = actuals_org_Q_2019.merge(pred_org_Q_2019, on="ds")
# plotting training, test and prediction data
org_Q_df.set_index("ds")["y"].plot(label="training")
org_Q_2019.set_index("ds")["y"].plot(label="test")
org_Q_2019.set_index("ds")["yhat"].plot(label="prediction")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - test vs. prediction")
plt.legend()
plt.savefig("Volume organic - test vs. prediction.png")
plt.show()
# model's MAPE calculation
model_org_Q_MAPE = MAPE(org_Q_2019["y"], org_Q_2019["yhat"])
# default vs. tuned models' MAPE
print("Default model MAPE: {:.2f}".format(model_org_Q_def_MAPE))
print("Tuned model MAPE: {:.2f}".format(model_org_Q_MAPE))
Default model MAPE: 27.94 Tuned model MAPE: 23.18
# simplifing the full dataset for use by Prophet
org_Q_full = prices_US_org_merged[["Date", "TotalVolume"]]
org_Q_full.columns = ["ds", "y"]
# re-training the model using the best performing parameters and full dataset
# creating 6-month (= 26-week) forecast
model_org_Q_for = Prophet(**best_params_org_Q)
model_org_Q_for.fit(org_Q_full)
future_6m_org_Q = model_org_Q_for.make_future_dataframe(periods=26, freq="W", include_history=False)
forecast_6m_org_Q = model_org_Q_for.predict(future_6m_org_Q)
fig_6m_org_Q = model_org_Q_for.plot(forecast_6m_org_Q)
WARNING:prophet.models:Optimization terminated abnormally. Falling back to Newton.
# plotting forecast
org_Q_full.set_index("ds")["y"].plot(label="original")
forecast_6m_org_Q.set_index("ds")["yhat"].plot(label="forecast")
plt.xlabel("Date")
plt.ylabel("Volume")
plt.title("Volume organic - forecast")
plt.legend()
plt.savefig("Volume organic - forecast.png")
plt.show()